daniel rectangle feature handle¶

import yaml

from torch.distributions.categorical import Categorical

# visualization

%matplotlib inline

import matplotlib.pyplot as plt

import warnings

from tqdm import tqdm_notebook

import torch.nn.functional as F

####### my own import file ##########

from listofpathpoint import input_handler

import cnc_input

import copy

#from hybrid_models import HPN

####### my own import file ##########

'''

so, the models we have are TransEncoderNet,

                            Attention

                            LSTM

                            HPN

each one have initial parameters and the forward part, 

once we have the forward part, the back propagation will 

finished automatically by pytorch  

'''

TOL = 1e-3

TINY = 1e-15

learning_rate = 1e-4   #learning rate

B = 128          #batch size

B_valLoop = 1

steps = 2500

n_epoch = 100       # epochs

map_number = 0

record_actor = []

record_critic = []

dimension = 4

speed_of_nozzle = 30

range_of_wait = 20

scale_of_the_map = (800,500,speed_of_nozzle*range_of_wait)

​

print('======================')

print('prepare to train')

print('======================')

print('Hyper parameters:')

print('learning rate', learning_rate)

print('batch size', B)

print('steps', steps)

print('epoch', n_epoch)

print('======================')

​

'''

instantiate a training network and a baseline network

'''

​

​

try:

    del Actor  # remove existing model

    del Critic # remove existing model

except:

    pass

Actor = HPN(n_feature = dimension, n_hidden = 128)

Critic = HPN(n_feature = dimension, n_hidden = 128)

optimizer = optim.Adam(Actor.parameters(), lr=learning_rate)

​

# Putting Critic model on the eval mode

Actor = Actor.to(device)

Critic = Critic.to(device)

Critic.eval()

​

epoch_ckpt = 0

tot_time_ckpt = 0

​

val_mean = []

val_std = []

maplist = ['10&15data/25_chips/25_1.json'

          ,'10&15data/25_chips/25_1.json','10&15data/25_chips/25_1.json'\

          ,'10&15data/25_chips/25_1.json','10&15data/25_chips/25_1.json'\

          ,'10&15data/25_chips/25_1.json'

          ]

plot_performance_train = []

plot_performance_baseline = []

# recording the result of the resent epoch makes it available for future

#*********************# Uncomment these lines to load the previous check point

"""

checkpoint_file = "checkpoint/mutimap_20.pkl"

checkpoint = torch.load(checkpoint_file, map_location=device)

epoch_ckpt = checkpoint['epoch'] + 1

tot_time_ckpt = checkpoint['tot_time']

plot_performance_train = checkpoint['plot_performance_train']

plot_performance_baseline = checkpoint['plot_performance_baseline']

Critic.load_state_dict(checkpoint['model_baseline'])

Actor.load_state_dict(checkpoint['model_train'])

optimizer.load_state_dict(checkpoint['optimizer'])

​

print('Re-start training with saved checkpoint file={:s}\n  Checkpoint at epoch= {:d} and time={:.3f}min\n'.format(checkpoint_file,epoch_ckpt-1,tot_time_ckpt/60))

"""

​

#***********************# Uncomment these lines to load the previous check point

​

# Main training loop

# The core training concept mainly upon Sampling from the actor

# then taking the greedy action from the critic

​

start_training_time = time.time()

time_stamp = datetime.datetime.now().strftime("%y-%m-%d--%H-%M-%S") # Load the time stamp

​

C = 0       # baseline => the object which the actor can compare

R = 0       # reward

C_critic = torch.Tensor([12000])

if torch.cuda.is_available():

    C_critic = C_critic.cuda()

temp = input_handler(maplist[map_number])

X_temp, mask_list_num = temp.final_ver_points()

print(X_temp)

print(mask_list_num)

#change the waiting time to the distance that we can measure

size = len(X_temp)

zero_to_bsz = torch.arange(B, device = device) # a list contains 0 to (batch size -1)

# load the GCN table

GCN_table = y_table.repeat(B,1,1)

​

for corner in X_temp:

    corner[dimension -2] *= speed_of_nozzle

size_rec = mask_list_num[-1]

print(size_rec)

​

X_temp = torch.FloatTensor(X_temp)

visit_count_initial = X_temp[:,3]

visit_count_rec = []

for i in range(0,len(visit_count_initial),4):

    visit_count_rec.append(int(visit_count_initial[i]))

X = X_temp.repeat(B,1,1)

if torch.cuda.is_available():

    X = X.cuda()

​

for epoch in range(0, n_epoch):

    # re-start training with saved checkpoint

    epoch += epoch_ckpt # adding the number of the former epochs

    # Train the model for one epoch

    start = time.time() # record the starting time

    Actor.train() 

    path_gazebo = []

    for i_all in range(1, steps+1): # 1 ~ 2500 steps

        # mask some points that are not the first visited points

        if torch.cuda.is_available():

            R = torch.zeros(B).cuda()

            reward_recorder = torch.zeros(B,size_rec).cuda()

            idx_recorder = torch.zeros(B,size_rec).cuda()

            logprobs = 0

            reward = torch.zeros(B).cuda()

            mask = torch.zeros(B,size).cuda() # use mask to make some points impossible to choose

        else:

            R = torch.zeros(B)

            reward_recorder = torch.zeros(B,size_rec)

            idx_recorder = torch.zeros(B,size_rec)

            logprobs = 0

            reward = torch.zeros(B)

            mask = torch.zeros(B,size) # use mask to make some points impossible to choose

        x = torch.zeros(B,dimension)# Y[:,0,:] #set the first point to x

        h = None

        c = None

        context = None        # set Y_ini to the out corner

        Transcontext = None

        Y0 = None

        choosen_idx = None

        visit_count = visit_count_initial.repeat(B,1)

        visit_time_count_initial = torch.zeros(len(visit_count_initial))

        visit_time_count = visit_time_count_initial.repeat(B,1)

        if torch.cuda.is_available():

            visit_count = visit_count.cuda()

        # Actor Sampling phase

        for k in range(size_rec):

            if k ==0:

                Y = X.view(B,size,dimension)

            context, Transcontext, output, h, c, _ = Actor(context,Transcontext,x=x, X_all=X,h=h,

                                                           c=c, mask=mask, choosen_idx=choosen_idx, GCN_table=GCN_table,if_check_u=False)

            sampler = torch.distributions.Categorical(output)

            idx = sampler.sample()

            idx_recorder[:,k] = (idx/4).type(torch.long)

            #prepare for the back propagation of pytorch

            Y, reward, Y0, x, choosen_idx = rectangle_process(temp, idx,Y,Y0,mask,k,B,i_all,path_gazebo,

                                                visit_count, visit_time_count, if_actor=True)

            R += reward

            reward_recorder[:,k] = R

            logprobs += torch.log(output[zero_to_bsz, idx.data] + TINY)

        #now is the time to check if any trajectory should be punished due to waiting on the same rectangle

​

        trajec_count = 0

        for path_time in zip(idx_recorder,reward_recorder):

            idx_time = zip(path_time[0],path_time[1])

            idx_time = sorted(idx_time, key=lambda x: x[0])

            total_idx = 0

            extra_waiting_time = 0

            for item in visit_count_rec:

                compare_list = []

                if item>1:

                    for i in range(int(item)):

                        compare_list.append(idx_time[int(total_idx + i)][1])

                    compare_list.sort(reverse=True)

                    total_idx += item

                    for i in range(len(compare_list)-1):

                        dis_step = compare_list[i] - compare_list[i+1]

                        dry_time = (range_of_wait/2)*speed_of_nozzle

                        if (dis_step) < (dry_time):

                            extra_waiting_time += dry_time - dis_step

                else:

                    total_idx += item

            R[trajec_count] += extra_waiting_time

            trajec_count+=1

# # critic baseline phase, use the baseline to compute the actual reward of agent at that time

#         if torch.cuda.is_available():

#             C = torch.zeros(B).cuda()

#             reward_recorder = torch.zeros(B,size_rec).cuda()

#             idx_recorder = torch.zeros(B,size_rec).cuda()

#             baseline = torch.zeros(B).cuda()

#             mask = torch.zeros(B,size).cuda() # use mask to make some points impossible to choose

​

#         else:

#             C = torch.zeros(B)

#             reward_recorder = torch.zeros(B,size_rec)

#             idx_recorder = torch.zeros(B,size_rec)

#             baseline = torch.zeros(B)

#             mask = torch.zeros(B,size) # use mask to make some points impossible to choose

#         x = torch.zeros(B,dimension)#Y[:,0,:] #set the first point to x

#         h = None

#         c = None

#         context = None

#         Transcontext = None

#         C0 = None

#         visit_count = visit_count_initial.repeat(B,1)

#         if torch.cuda.is_available():

#             visit_count = visit_count.cuda()

#         # compute tours for baseline without grad "Cause we want to fix the weights for the critic"

#         with torch.no_grad():

#             for k in range(size_rec):     

#                 Y = X.view(B,size,dimension)

#                 #check if all the points to be masked,if so, raise the total reward R, and check again

#                 context, Transcontext, output, h, c, _ = Critic(context,Transcontext,x=x, X_all=X,h=h,

#                                                                 c=c, mask=mask)

#                 idx = torch.argmax(output, dim=1) # ----> greedy baseline critic

#                 idx_recorder[:,k] = (idx/4).type(torch.long)

#                 # prepare for the back propagation of pytorch

#                 baseline, C0,x = rectangle_process(temp,idx,Y,C0, mask,k,B,i_all,path_gazebo,

#                                                    visit_count,if_actor=False)

#                 C += baseline

#                 reward_recorder[:,k] = C

#             #now is the time to check if any trajectory should be punished due to waiting on the same rectangle

​

#             trajec_count = 0

#             for path_time in zip(idx_recorder,reward_recorder):

#                 idx_time = zip(path_time[0],path_time[1])

#                 idx_time = sorted(idx_time, key=lambda x: x[0])

#                 total_idx = 0

#                 extra_waiting_time = 0

#                 for item in visit_count_rec:

#                     compare_list = []

#                     if item>1:

#                         for i in range(int(item)):

#                             compare_list.append(idx_time[int(total_idx + i)][1])

#                         compare_list.sort(reverse=True)

#                         total_idx += item

#                         for i in range(len(compare_list)-1):

#                             dis_step = compare_list[i] - compare_list[i+1]

#                             dry_time = (range_of_wait/2)*speed_of_nozzle

#                             if (dis_step) < (dry_time):

#                                 extra_waiting_time += dry_time - dis_step

#                     else:

#                         total_idx += item

#                 C[trajec_count] += extra_waiting_time

#                 trajec_count+=1

        ###################

        # Loss and backprop handling 

        ###################

        loss = torch.mean((R - C_critic[0]) * logprobs)

        optimizer.zero_grad()

        loss.backward()

        optimizer.step()

        if i_all % 100 == 0:

            print("epoch:{}, batch:{}/{}, reward:{}".format(epoch, i_all, steps,R.mean().item()))

            record_actor.append(R.mean().tolist())

            record_critic.append(C_critic[0].tolist())

            plt.plot(record_actor,'r:')

            plt.plot(record_critic,'b:')

            plt.show()

        if i_all % 100 == 0:

            print("record the last path to gazebo for showing up")

            #starting to show the path on simulated enviroment of cnc_machine 

            the_resent_path = temp.zig_zag_path(path_gazebo,mask_list_num)

            data = {'path':the_resent_path}

            data_1 = {'corners':path_gazebo}

            pathpoints_dir = os.path.join("pathpoints")

            if not os.path.exists(pathpoints_dir):

                os.makedirs(pathpoints_dir)

            name = 'pathpoints/path_points '+str(i)+'.yaml'

            with open(name, 'w') as file:

                documents = yaml.dump(data,file)

                documents = yaml.dump(data_1,file)

            path_gazebo = []

    time_one_epoch = time.time() - start #recording the work time of one epoch

    time_tot = time.time() - start_training_time + tot_time_ckpt

    ###################

    # Evaluate train model and baseline 

    # in this phase we just solve random instances with the actor and the critic

    # compare this soluation if we get any improvment we'll transfer the actor's

    # weights into the critic

    ###################

    # putting the actor in the eval mode

    Actor.eval()

    mean_tour_length_actor = 0

    mean_tour_length_critic = 0

​

    for step in range(0,B_valLoop):

        # compute tour for model and baseline

        # mask some points that are not the first visited points

        if torch.cuda.is_available():

            R = torch.zeros(B).cuda()

            reward_recorder = torch.zeros(B,size_rec).cuda()

            idx_recorder = torch.zeros(B,size_rec).cuda()

            reward = torch.zeros(B).cuda()

            mask = torch.zeros(B,size).cuda() # use mask to make some points impossible to choose

        else:

            R = torch.zeros(B)

            reward_recorder = torch.zeros(B,size_rec)

            idx_recorder = torch.zeros(B,size_rec)

            reward = torch.zeros(B)

            mask = torch.zeros(B,size) # use mask to make some points impossible to choose

        x = torch.zeros(B,dimension)# Y[:,0,:] #set the first point to x

        h = None

        c = None

        context = None        # set Y_ini to the out corner

        Transcontext = None

        Y0 = None

        choosen_idx = None

        visit_count = visit_count_initial.repeat(B,1)

        visit_time_count_initial = torch.zeros(len(visit_count_initial))

        visit_time_count = visit_time_count_initial.repeat(B,1)

        if torch.cuda.is_available():

            visit_count = visit_count.cuda()

​

        with torch.no_grad():

            for k in range(size_rec):

                if k==0:

                    Y = X.view(B,size,dimension)

                context, Transcontext, output, h, c, _ = Actor(context,Transcontext,x=x, X_all=X,h=h,

                                                               c=c, mask=mask,choosen_idx=choosen_idx,GCN_table=GCN_table,if_check_u=True)

                idx = torch.argmax(output, dim=1)

                idx_recorder[:,k] = (idx/4).type(torch.long)

                #prepare for the back propagation of pytorch

                Y, reward, Y0, x, choosen_idx = rectangle_process(temp, idx,Y,Y0,mask,k,B,i,path_gazebo,

                                                    visit_count, visit_time_count, if_actor=False)

                R += reward

                reward_recorder[:,k] = R

            #now is the time to check if any trajectory should be punished due to waiting on the same rectangle

​

            trajec_count = 0

            for path_time in zip(idx_recorder,reward_recorder):

                idx_time = zip(path_time[0],path_time[1])

                idx_time = sorted(idx_time, key=lambda x: x[0])

                total_idx = 0

                extra_waiting_time = 0

                for item in visit_count_rec:

                    compare_list = []

                    if item>1:

                        for i in range(int(item)):

                            compare_list.append(idx_time[int(total_idx + i)][1])

                        compare_list.sort(reverse=True)

                        total_idx += item

                        for i in range(len(compare_list)-1):

                            dis_step = compare_list[i] - compare_list[i+1]

                            dry_time = (range_of_wait/2)*speed_of_nozzle

                            if (dis_step) < (dry_time):

                                extra_waiting_time += dry_time - dis_step

                    else:

                        total_idx += item

                R[trajec_count] += extra_waiting_time

                trajec_count+=1

            print('R_val = ',R[0])

        if torch.cuda.is_available():

            C = torch.zeros(B).cuda()

            reward_recorder = torch.zeros(B,size_rec).cuda()

            idx_recorder = torch.zeros(B,size_rec).cuda()

            baseline = torch.zeros(B).cuda()

            mask = torch.zeros(B,size).cuda() # use mask to make some points impossible to choose

        else:

            C = torch.zeros(B)

            reward_recorder = torch.zeros(B,size_rec)

            idx_recorder = torch.zeros(B,size_rec)

            baseline = torch.zeros(B)

            mask = torch.zeros(B,size) # use mask to make some points impossible to choose

        x = torch.zeros(B,dimension)#Y[:,0,:] #set the first point to x

        h = None

        c = None

        context = None

        Transcontext = None

        C0 = None

        choosen_idx = None

        visit_count = visit_count_initial.repeat(B,1)

        visit_time_count_initial = torch.zeros(len(visit_count_initial))

        visit_time_count = visit_time_count_initial.repeat(B,1)

        if torch.cuda.is_available():

            visit_count = visit_count.cuda()

        # compute tours for baseline without grad "Cause we want to fix the weights for the critic"

        with torch.no_grad():

            for k in range(size_rec): 

                if k ==0:

                    Y = X.view(B,size,dimension)

                #check if all the points to be masked,if so, raise the total reward R, and check again

                context, Transcontext, output, h, c, _ = Critic(context,Transcontext,x=x, X_all=X,h=h,

                                                                c=c, mask=mask, choosen_idx=choosen_idx, GCN_table=GCN_table, if_check_u=False)

                idx = torch.argmax(output, dim=1) # ----> greedy baseline critic